<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2014-2016 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Hadoop Compatibility</title>

    <link rel="stylesheet" href="_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="_static/jquery.js"></script>
    <script type="text/javascript" src="_static/underscore.js"></script>
    <script type="text/javascript" src="_static/doctools.js"></script>
    <script type="text/javascript" src="_static/language_data.js"></script>

    <link rel="shortcut icon" href="_static/favicon.ico"/>
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="index.html" />
    <link rel="next" title="CDAP and Hadoop Compatibility" href="cdap-hadoop-compatibility.html" />
    <link rel="prev" title="Deployment Architectures" href="deployment-architectures.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: admin-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><a href="table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><b><a href="table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="index.html"> Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="cdap-components.html"> CDAP Components</a></li>
<li class="toctree-l1"><a class="reference internal" href="deployment-architectures.html"> Deployment Architectures</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#"> Hadoop Compatibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="cdap-hadoop-compatibility.html"> CDAP and Hadoop Compatibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="system-requirements.html"> System Requirements</a></li>
<li class="toctree-l1"><a class="reference internal" href="installation/index.html"> Installation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="installation/cloudera.html">Cloudera Manager</a></li>
<li class="toctree-l2"><a class="reference internal" href="installation/emr.html">Amazon EMR</a></li>
<li class="toctree-l2"><a class="reference internal" href="installation/ambari.html">Apache Ambari</a></li>
<li class="toctree-l2"><a class="reference internal" href="installation/mapr.html">MapR</a></li>
<li class="toctree-l2"><a class="reference internal" href="installation/azure-hdinsight.html">Microsoft Azure HDInsight</a></li>
<li class="toctree-l2"><a class="reference internal" href="installation/packages.html">Packages</a></li>
<li class="toctree-l2"><a class="reference internal" href="installation/replication.html">Replication</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="incompatibilities.html"> Incompatibilities</a></li>
<li class="toctree-l1"><a class="reference internal" href="upgrading/index.html"> Upgrading</a><ul>
<li class="toctree-l2"><a class="reference internal" href="upgrading/cloudera.html">Cloudera Manager</a></li>
<li class="toctree-l2"><a class="reference internal" href="upgrading/ambari.html">Apache Ambari</a></li>
<li class="toctree-l2"><a class="reference internal" href="upgrading/mapr.html">MapR</a></li>
<li class="toctree-l2"><a class="reference internal" href="upgrading/packages.html">Packages</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="security/index.html"> Security</a><ul>
<li class="toctree-l2"><a class="reference internal" href="security/perimeter-security.html">Perimeter Security</a></li>
<li class="toctree-l2"><a class="reference internal" href="security/authorization.html">Authorization</a></li>
<li class="toctree-l2"><a class="reference internal" href="security/impersonation.html">Impersonation</a></li>
<li class="toctree-l2"><a class="reference internal" href="security/system-services.html">Enabling SSL for System Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="security/secure-storage.html">Secure Storage</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="operations/index.html"> Operations</a><ul>
<li class="toctree-l2"><a class="reference internal" href="operations/logging.html"> Logging and Monitoring</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations/metrics.html"> Metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations/operations-dashboard.html"> Dashboard and Reports</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations/preferences.html"> Preferences and Runtime Arguments</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations/scaling-instances.html"> Scaling Instances</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations/resource-guarantees.html"> Resource Guarantees in YARN</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations/tx-maintenance.html"> Transaction Service Maintenance</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations/cdap-ui.html"> CDAP UI</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="appendices/index.html"> Appendices</a><ul>
<li class="toctree-l2"><a class="reference internal" href="appendices/cdap-site.html"> Appendix: cdap-site.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="appendices/cdap-security.html"> Appendix: cdap-security.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="appendices/minimal-cdap-site.html"> Appendix: Minimal cdap-site.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="appendices/hbase-ddl-executor.html"> Appendix: HBaseDDLExecutor</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="hadoop-compatibility">
<span id="admin-manual-hadoop-compatibility-matrix"></span><h1>Hadoop Compatibility<a class="headerlink" href="#hadoop-compatibility" title="Permalink to this headline">🔗</a></h1>
<p>Before installing the CDAP components, you must first install (or have access to) a Hadoop
cluster with <em>HBase</em>, <em>HDFS</em>, <em>YARN</em>, and <em>ZooKeeper</em>. <em>Hive</em> and <em>Spark</em> are optional
components; Hive is required to enable CDAP’s ad-hoc querying capabilities (<em>CDAP
Explore</em>) and Spark is required if a CDAP application uses the Spark program.</p>
<p>All CDAP components can be installed on the same boxes as your Hadoop cluster, or on
separate boxes that can connect to the Hadoop services.</p>
<p>CDAP depends on these services being present on the cluster. There are <strong>core
dependencies,</strong> which must be running for CDAP system services to operate correctly, and
<strong>optional dependencies,</strong> which may be required for certain functionality or program types.</p>
<p>The host(s) running the CDAP Master service must have the HBase, HDFS, and YARN clients
installed, as CDAP uses the command line clients of these for initialization and their
connectivity information for external service dependencies. If Hadoop system services are
also running on the same hosts as the CDAP services, they will already have these clients
installed.</p>
<p><strong>Core Dependencies</strong></p>
<ul class="simple">
<li><strong>HBase:</strong> For system runtime storage and queues</li>
<li><strong>HDFS:</strong> The backing file system for distributed storage</li>
<li><strong>YARN:</strong> For running system services in containers on cluster NodeManagers</li>
<li><strong>MapReduce2:</strong> For batch operations in workflows and data exploration (included with YARN)</li>
<li><strong>ZooKeeper:</strong> For service discovery and leader election</li>
</ul>
<p><strong>Optional Dependencies</strong></p>
<ul class="simple">
<li><strong>Hive:</strong> For data exploration using SQL queries via the CDAP Explore system service</li>
<li><strong>Spark:</strong> For running Spark programs within CDAP applications</li>
</ul>
<p class="rubric">Hadoop/HBase Environment</p>
<p id="admin-manual-hadoop-compatibility-matrix-required">For a Distributed CDAP cluster, version 6.1.1, you <strong>must</strong> install these Hadoop components
(see notes following the tables):</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="38%" />
<col width="44%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Component</th>
<th class="head">Source</th>
<th class="head">Supported Versions</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><strong>Hadoop</strong></td>
<td><em>various</em></td>
<td>2.0 and higher</td>
</tr>
<tr class="row-odd"><td rowspan="5"><strong>HBase</strong></td>
<td>Apache</td>
<td>0.98.x and 1.2</td>
</tr>
<tr class="row-even"><td>Cloudera Distribution of Apache Hadoop (CDH)</td>
<td>5.1 through 5.12 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-odd"><td>Hortonworks Data Platform (HDP)</td>
<td>2.0 through 2.6 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-even"><td>MapR</td>
<td>5.0 through 5.2 (with Apache HBase)</td>
</tr>
<tr class="row-odd"><td>Amazon Hadoop (EMR)</td>
<td>4.6 through 4.8 (with Apache HBase)</td>
</tr>
<tr class="row-even"><td rowspan="5"><strong>HDFS</strong></td>
<td>Apache Hadoop</td>
<td>2.0.2-alpha through 2.6</td>
</tr>
<tr class="row-odd"><td>Cloudera Distribution of Apache Hadoop (CDH)</td>
<td>5.1 through 5.12 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-even"><td>Hortonworks Data Platform (HDP)</td>
<td>2.0 through 2.6 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-odd"><td>MapR</td>
<td>5.0 through 5.2 (with MapR-FS)</td>
</tr>
<tr class="row-even"><td>Amazon Hadoop (EMR)</td>
<td>4.6 through 4.8</td>
</tr>
<tr class="row-odd"><td rowspan="5"><strong>YARN</strong> and
<strong>MapReduce2</strong></td>
<td>Apache Hadoop</td>
<td>2.0.2-alpha through 2.7</td>
</tr>
<tr class="row-even"><td>Cloudera Distribution of Apache Hadoop (CDH)</td>
<td>5.1 through 5.12 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-odd"><td>Hortonworks Data Platform (HDP)</td>
<td>2.0 through 2.6 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-even"><td>MapR</td>
<td>5.0 through 5.2</td>
</tr>
<tr class="row-odd"><td>Amazon Hadoop (EMR)</td>
<td>4.6 through 4.8</td>
</tr>
<tr class="row-even"><td rowspan="5"><strong>ZooKeeper</strong></td>
<td>Apache</td>
<td>Version 3.4.3 through 3.4</td>
</tr>
<tr class="row-odd"><td>Cloudera Distribution of Apache Hadoop (CDH)</td>
<td>5.1 through 5.12 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-even"><td>Hortonworks Data Platform (HDP)</td>
<td>2.0 through 2.6 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-odd"><td>MapR</td>
<td>5.0 through 5.2</td>
</tr>
<tr class="row-even"><td>Amazon Hadoop (EMR)</td>
<td>4.6 through 4.8</td>
</tr>
</tbody>
</table>
<p id="admin-manual-hadoop-compatibility-matrix-optional">For a Distributed CDAP cluster, version 6.1.1, you <em>can</em> (optionally) install these
Hadoop components, as required:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="38%" />
<col width="44%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Component</th>
<th class="head">Source</th>
<th class="head">Supported Versions</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td rowspan="5"><strong>Hive</strong></td>
<td>Apache</td>
<td>Version 0.12.0 through 1.2.x</td>
</tr>
<tr class="row-odd"><td>Cloudera Distribution of Apache Hadoop (CDH)</td>
<td>5.1 through 5.12 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-even"><td>Hortonworks Data Platform (HDP)</td>
<td>2.0 through 2.6 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-odd"><td>MapR</td>
<td>5.0 through 5.2</td>
</tr>
<tr class="row-even"><td>Amazon Hadoop (EMR)</td>
<td>4.6 through 4.8</td>
</tr>
<tr class="row-odd"><td rowspan="5"><strong>Spark</strong></td>
<td>Apache</td>
<td>Versions 1.6.x through 2.3.x</td>
</tr>
<tr class="row-even"><td>Cloudera Distribution of Apache Hadoop (CDH)</td>
<td>5.1 through 5.12 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-odd"><td>Hortonworks Data Platform (HDP)</td>
<td>2.0 through 2.6.5 <a class="reference internal" href="#admin-manual-hadoop-compatibility-matrix-note-4"><span class="std std-ref">(Note 4)</span></a></td>
</tr>
<tr class="row-even"><td>MapR</td>
<td>5.0 through 5.2</td>
</tr>
<tr class="row-odd"><td>Amazon Hadoop (EMR)</td>
<td>4.6 through 4.8</td>
</tr>
</tbody>
</table>
<p><strong>Note 1:</strong> Component versions shown in these tables are those that we have tested and are
confident of their suitability and compatibility. Later versions of components may work,
but have not necessarily been either tested or confirmed compatible.</p>
<p><strong>Note 2:</strong> Certain CDAP components need to reference your <em>Hadoop</em>, <em>YARN</em>, <em>HBase</em>, and
<em>Hive</em> cluster configurations by adding those configurations to their class paths.</p>
<p><strong>Note 3:</strong> <em>Hive 0.12</em> is not supported for <a class="reference internal" href="security/index.html#admin-security"><span class="std std-ref">secure cluster configurations</span></a>.</p>
<p id="admin-manual-hadoop-compatibility-matrix-note-4"><strong>Note 4:</strong> An upcoming release of CDAP (scheduled for CDAP 4.3) will drop support for all
versions older than <a class="reference external" href="https://www.cloudera.com/documentation/other/security-bulletins/topics/csb_all_product_issues.html#tsb_175">CDH 5.4.11</a>
or <a class="reference external" href="http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.5.0/bk_release-notes/content/ch_relnotes_v250.html">HDP 2.5.0.0</a>
due to an <a class="reference external" href="https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-5393">Apache Hadoop Privilege Escalation Vulnerability</a>.</p>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Deployment Architectures" href="deployment-architectures.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="CDAP and Hadoop Compatibility" href="cdap-hadoop-compatibility.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="_static/js/js.cookie.js"></script><script type="text/javascript" src="_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>